import json
import random
import sys
import os
from urllib.parse import urlencode

# 获取当前文件所在目录的父目录（即项目根目录）
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
if project_root not in sys.path:
    sys.path.append(project_root)

import re
import threading
import time
from concurrent.futures.thread import ThreadPoolExecutor
from datetime import datetime, timedelta
import requests
import yaml
from requests.exceptions import ProxyError, Timeout, SSLError
import httpx
from httpx import Response
from common import feishu_api
from common.oss import OSSManager, get_token
from common.start import INSERT_TRINO_LOCK, get_random_proxy, douyin_logger, add_proxy_to_blacklist, config, THIS_UPDATE_TIME

from common.trino_client import  get_creators_by_dt_media, get_previous_day_update_creators_by_media, get_tags_by_id_and_media, insert_douyin_user_detail, insert_douyin_video_details, insert_success_log_by_media, update_tags_by_id_and_media
from douyin.utils import BogusManager, gen_a_bogus, get_a_bogus_from_js, get_timestamp

DEFAULT_PARAMS = {
    'device_platform': "webapp",
    'aid': '6383',
    'channel': 'channel_pc_web',
    "pc_client_type": 1,
    "version_code": "290100",
    "version_name": "29.1.0",
    "cookie_enabled": "true",
    "screen_width": 1920,
    "screen_height": 1080,
    "browser_language": "zh-CN",
    "browser_platform": "Win32",
    "browser_name": "Chrome",
    "browser_version":  "130.0.0.0",
    "browser_online": "true",
    "engine_name": "Blink",
    "engine_version": "130.0.0.0",
    "os_name": "Windows",
    "os_version": "10",
    "cpu_core_num": 12,
    "device_memory":  8,
    "platform": "PC",
    "downlink": "10",
    "effective_type": "4g",
    "from_user_page": "1",
    "locate_query": "false",
    "need_time_list": "1",
    "pc_libra_divert": "Windows",
    "publish_video_strategy_type": "2",
    "round_trip_time": "0",
    "show_live_replay_strategy": "1",
    "time_list_query": "0",
    "whale_cut_token": "",
    "update_version_code":  "170400",
}

with open(f"./douyin/douyin_config.yaml", "r", encoding="utf-8") as f:
    config_douyin = yaml.safe_load(f)


def get_douyin_headers():
    douyin_config = config_douyin["TokenManager"]["douyin"]
    kwargs = {
        "headers": {
            "Accept-Language": douyin_config["headers"]["Accept-Language"],
            "User-Agent": douyin_config["headers"]["User-Agent"],
            "Referer": douyin_config["headers"]["Referer"],
            "Cookie": douyin_config["headers"]["Cookie"],
        }
    }
    return kwargs

def get_search_id(keyword):
    max_retries = 3
    retries = 0
    while retries < max_retries:
        proxy = get_random_proxy()
        douyin_logger.info(f"Using proxy: {proxy}")
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            kwargs = get_douyin_headers()
            headers = kwargs["headers"]
            url = "https://www.douyin.com/aweme/v1/web/search/item/"
            params = {
                'aid': '6383',
                'channel': 'channel_pc_web',
                'search_channel': 'aweme_video_web',
                'enable_history': '1',
                'keyword': keyword,
                'search_source': 'normal_search',
                'query_correct_type': '1',
                'is_filter_search': '0',
                'from_group_id': '',
                'offset': 0,
                'count': 10,
                'need_filter_settings': '1',
                'list_type': 'multi',
                # 'search_id': '20250417110024B4284B3E2992B165E1E8',
                'update_version_code': '170400',
                'pc_client_type': '1',
                'version_code': '190600',
                'version_name': '19.6.0',
                'cookie_enabled': 'true',
                'screen_width': '1536',
                'screen_height': '864',
                'browser_language': 'zh-CN',
                'browser_platform': 'Win32',
                'browser_name': 'Edge',
                'browser_version': '124.0.0.0',
                'browser_online': 'true',
                'engine_name': 'Blink',
                'engine_version': '124.0.0.0',
                'os_name': 'Windows',
                'os_version': '10',
                'cpu_core_num': '8',
                'device_memory': '8',
                'platform': 'PC',
                'downlink': '10',
                'effective_type': '4g',
                'round_trip_time': '50',
                'webid': '7372443638851700278',
                'msToken': 'gPazFU7zvRy2fx5JYWnXaR3yCzwi2iXSbUJHiGGkyXoW8NIJ06cGTwyQsEzkVpmfr9uZfsNhhctq_k_byR87aMpd9yNkSI0diSAw6Nfm62H0PWjRKevlqjenVI7K7w==',
                'a_bogus': 'mXmZMdzgdDfiDDWX5VcLfY3q6WB3Y/R30CPYMD2f7dVrng39HMOH9exoM3zvDpRjLG/lIeujy4hbO3KprQVHMZwf9W0L/252sESkKl5Q5xSSs1X9eghgJ0hwmkt5SMx2RvB-rOXgqwlHFmym09oHmhK4bIOwu3GMEE==',
            }
            headers["Cookie"] = ""
            response = requests.get(url, headers=headers, params=params, proxies=proxies)
            # print(response.json())
            douyin_logger.info(f"Response status code: {response.status_code}")
            if response.json()["status_code"] == 2484:
                douyin_logger.error(f"status_code = {2484}")
            douyin_logger.info(f"获取search_id成功:{response.json().get('log_pb').get('impr_id')}")
            return response.json().get("log_pb").get("impr_id")
        except (ProxyError, ConnectionError, Timeout, SSLError) as e:
            douyin_logger.warning(f"Proxy connection failed ({type(e).__name__})")
            add_proxy_to_blacklist(proxy["ip"])
            retries += 1
            time.sleep(1)  # 失败后等待1秒再重试
        except Exception as e:
            douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
            retries += 1
            


def get_play_count_by_aweme_id(aweme_id):
    max_retries = 3
    retries = 0
    while retries < max_retries:
        proxy = get_random_proxy()
        douyin_logger.info(f"Using proxy: {proxy}")
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            kwargs = get_douyin_headers()
            headers = kwargs["headers"]
            url = "https://www-hj.douyin.com/aweme/v1/web/mix/aweme/"
            params = {
                'aid': '6383',
                'channel': 'channel_pc_web',
                'mix_id': '7481241263842854950',
                'cursor': '11',
                'count': '6',
                # 'search_id': '20250417110024B4284B3E2992B165E1E8',
                'update_version_code': '170400',
                'pc_client_type': '1',
                'version_code': '190600',
                'version_name': '19.6.0',
                'cookie_enabled': 'true',
                'screen_width': '1536',
                'screen_height': '864',
                'browser_language': 'zh-CN',
                'browser_platform': 'Win32',
                'browser_name': 'Edge',
                'browser_version': '124.0.0.0',
                'browser_online': 'true',
                'engine_name': 'Blink',
                'engine_version': '124.0.0.0',
                'os_name': 'Windows',
                'os_version': '10',
                'cpu_core_num': '8',
                'device_memory': '8',
                'platform': 'PC',
                'downlink': '10',
                'effective_type': '4g',
                'round_trip_time': '50',
                'webid': '7372443638851700278',
                'msToken': 'gPazFU7zvRy2fx5JYWnXaR3yCzwi2iXSbUJHiGGkyXoW8NIJ06cGTwyQsEzkVpmfr9uZfsNhhctq_k_byR87aMpd9yNkSI0diSAw6Nfm62H0PWjRKevlqjenVI7K7w==',
                'a_bogus': 'mXmZMdzgdDfiDDWX5VcLfY3q6WB3Y/R30CPYMD2f7dVrng39HMOH9exoM3zvDpRjLG/lIeujy4hbO3KprQVHMZwf9W0L/252sESkKl5Q5xSSs1X9eghgJ0hwmkt5SMx2RvB-rOXgqwlHFmym09oHmhK4bIOwu3GMEE==',
            }
            headers["Cookie"] = ""
            response = requests.get(url, headers=headers, params=params, proxies=proxies)
            print(response.json())
            douyin_logger.info(f"Response status code: {response.status_code}")
            if response.json()["status_code"] == 2484:
                douyin_logger.error(f"status_code = {2484}")
            douyin_logger.info(f"获取search_id成功:{response.json().get('log_pb').get('impr_id')}")
            return response.json().get("log_pb").get("impr_id")
        except (ProxyError, ConnectionError, Timeout, SSLError) as e:
            douyin_logger.warning(f"Proxy connection failed ({type(e).__name__})")
            add_proxy_to_blacklist(proxy["ip"])
            retries += 1
            time.sleep(1)  # 失败后等待1秒再重试
        except Exception as e:
            douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
            retries += 1


def get_top100_posts_by_keyword_normal(keyword):
    post_list = []
    page = 0
    max_retries = 6
    number = 0
    p = 0
    has_more = True
    while len(post_list) < 100 and has_more:
        print(f"page:{page}")
        retries = 0
        while retries < max_retries:
            proxy = get_random_proxy()
            douyin_logger.info(f"Using proxy: {proxy}")
            try:
                proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
                proxies = {'http': proxy_url, 'https': proxy_url}
                kwargs = get_douyin_headers()
                headers = kwargs["headers"]
                url = "https://www.douyin.com/aweme/v1/web/search/item/"
                params = {
                    'aid': '6383',
                    'channel': 'channel_pc_web',
                    'search_channel': 'aweme_video_web',
                    'enable_history': '1',
                    'keyword': keyword,
                    'search_source': 'normal_search',
                    'query_correct_type': '1',
                    'is_filter_search': '0',
                    'from_group_id': '',
                    'offset': 10 + page * 10,
                    'count': 10,
                    'need_filter_settings': '1',
                    'list_type': 'multi',
                    # 'search_id': search_id,
                    'update_version_code': '170400',
                    'pc_client_type': '1',
                    'version_code': '190600',
                    'version_name': '19.6.0',
                    'cookie_enabled': 'true',
                    'screen_width': '1536',
                    'screen_height': '864',
                    'browser_language': 'zh-CN',
                    'browser_platform': 'Win32',
                    'browser_name': 'Edge',
                    'browser_version': '124.0.0.0',
                    'browser_online': 'true',
                    'engine_name': 'Blink',
                    'engine_version': '124.0.0.0',
                    'os_name': 'Windows',
                    'os_version': '10',
                    'cpu_core_num': '8',
                    'device_memory': '8',
                    'platform': 'PC',
                    'downlink': '10',
                    'effective_type': '4g',
                    'round_trip_time': '50',
                    'webid': '7372443638851700278',
                    'msToken': 'gPazFU7zvRy2fx5JYWnXaR3yCzwi2iXSbUJHiGGkyXoW8NIJ06cGTwyQsEzkVpmfr9uZfsNhhctq_k_byR87aMpd9yNkSI0diSAw6Nfm62H0PWjRKevlqjenVI7K7w==',
                    'a_bogus': 'mXmZMdzgdDfiDDWX5VcLfY3q6WB3Y/R30CPYMD2f7dVrng39HMOH9exoM3zvDpRjLG/lIeujy4hbO3KprQVHMZwf9W0L/252sESkKl5Q5xSSs1X9eghgJ0hwmkt5SMx2RvB-rOXgqwlHFmym09oHmhK4bIOwu3GMEE==',
                }
                headers["Cookie"] = ""
                print("------------------------------------")
                print(f"offset:{params['offset']}")
                # print(f"search_id:{params['search_id']}")
                response = requests.get(url, headers=headers, params=params, proxies=proxies,timeout=4)
                print(response.json())
                douyin_logger.info(f"Response status code: {response.status_code}")
                if response.json()["has_more"] == 0 and number != 0:
                    has_more = False
                    douyin_logger.info("没有更多数据了，结束请求")
                    break
                if len(response.json()["data"]) == 0:
                    douyin_logger.info("Search id 失效，重新获取......")
                    # search_id = get_search_id(keyword)
                    retries += 1
                    continue
                for post in response.json()["data"]:
                    cur = {
                        "desc": post["aweme_info"]["desc"],
                        "aweme_type": post["aweme_info"]["aweme_type"],
                        "nickname": post["aweme_info"]["author"]["nickname"],
                        "sec_uid": post["aweme_info"]["author"]["sec_uid"],
                        "statistics": post["aweme_info"]["statistics"],
                    }
                    post_list.append(cur)
                number = 1
                print(len(post_list))
                print("------------------------------------")
                break
            except (ProxyError, ConnectionError, Timeout, SSLError) as e:
                douyin_logger.warning(f"Proxy connection failed ({type(e).__name__})")
                add_proxy_to_blacklist(proxy["ip"])
                retries += 1
                time.sleep(1)  # 失败后等待1秒再重试
            except Exception as e:
                douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
                retries += 1
                break
        p += 1
        page += 1
        if not has_more or p > 15:
            return post_list
    print(len(post_list))
    return post_list[:100]

def get_top100_posts_by_keyword(keyword):
    post_list = []
    page = 1
    max_retries = 6
    number = 0
    p = 0
    has_more = True
    search_id = get_search_id(keyword)
    while len(post_list) < 100 and has_more:
        print(f"page:{page}")
        retries = 0
        while retries < max_retries:
            proxy = get_random_proxy()
            douyin_logger.info(f"Using proxy: {proxy}")
            try:
                proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
                proxies = {'http': proxy_url, 'https': proxy_url}
                kwargs = get_douyin_headers()
                headers = kwargs["headers"]
                url = "https://www.douyin.com/aweme/v1/web/search/item/"
                params = {
                    'aid': '6383',
                    'channel': 'channel_pc_web',
                    'search_channel': 'aweme_video_web',
                    'enable_history': '1',
                    'keyword': keyword,
                    'sort_type': '1',
                    'publish_time': '0',
                    'search_source': 'tab_search',
                    'query_correct_type': '1',
                    'is_filter_search': '1',
                    'from_group_id': '',
                    'offset': 0 + page * 10,
                    'count': 10,
                    'need_filter_settings': '1',
                    'list_type': 'multi',
                    'search_id': search_id,
                    'update_version_code': '170400',
                    'pc_client_type': '1',
                    'version_code': '190600',
                    'version_name': '19.6.0',
                    'cookie_enabled': 'true',
                    'screen_width': '1536',
                    'screen_height': '864',
                    'browser_language': 'zh-CN',
                    'browser_platform': 'Win32',
                    'browser_name': 'Edge',
                    'browser_version': '124.0.0.0',
                    'browser_online': 'true',
                    'engine_name': 'Blink',
                    'engine_version': '124.0.0.0',
                    'os_name': 'Windows',
                    'os_version': '10',
                    'cpu_core_num': '8',
                    'device_memory': '8',
                    'platform': 'PC',
                    'downlink': '10',
                    'effective_type': '4g',
                    'round_trip_time': '50',
                    'webid': '7372443638851700278',
                }
                headers["Cookie"] = ""
                print("------------------------------------")
                print(f"offset:{params['offset']}")
                print(f"search_id:{params['search_id']}")
                response = requests.get(url, headers=headers, params=params, proxies=proxies,timeout=4)
                # print(response.json())
                douyin_logger.info(f"Response status code: {response.status_code}")
                if response.json()["has_more"] == 0 and number != 0:
                    has_more = False
                    douyin_logger.info("没有更多数据了，结束请求")
                    break
                if len(response.json()["data"]) == 0:
                    douyin_logger.info("Search id 失效，重新获取......")
                    search_id = get_search_id(keyword)
                    retries += 1
                    continue
                for post in response.json()["data"]:
                    cur = {
                        "desc": post["aweme_info"]["desc"],
                        "aweme_type": post["aweme_info"]["aweme_type"],
                        "nickname": post["aweme_info"]["author"]["nickname"],
                        "sec_uid": post["aweme_info"]["author"]["sec_uid"],
                        "statistics": post["aweme_info"]["statistics"],
                    }
                    post_list.append(cur)
                number = 1
                search_id = response.json().get("log_pb").get("impr_id")
                print(len(post_list))
                print("------------------------------------")
                break
            except (ProxyError, ConnectionError, Timeout, SSLError) as e:
                douyin_logger.warning(f"Proxy connection failed ({type(e).__name__})")
                add_proxy_to_blacklist(proxy["ip"])
                retries += 1
                time.sleep(1)  # 失败后等待1秒再重试
            except Exception as e:
                douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
                retries += 1

        p += 1
        page += 1
        if not has_more or p > 15:
            return post_list
    print(len(post_list))
    return post_list[:100]


def get_top100_authors_by_keyword(keyword):
    posts_list = get_top100_posts_by_keyword(keyword)
    authors_dict = {}
    for post in posts_list:
        sec_uid = post.get("sec_uid")
        authors_dict[sec_uid] = {
            'sec_uid' : sec_uid,
            "nickname" : post.get("nickname"),
        }

    print(authors_dict.values())
    return list(authors_dict.values())

def get_top100_authors_by_keyword_normal(keyword):
    posts_list = get_top100_posts_by_keyword_normal(keyword)
    authors_dict = {}
    for post in posts_list:
        sec_uid = post.get("sec_uid")
        authors_dict[sec_uid] = {
            'sec_uid' : sec_uid,
            "nickname" : post.get("nickname"),
        }
    print(authors_dict.values())
    return list(authors_dict.values())

COOKIES = config_douyin["TokenManager"]["douyin"]["headers"]["Cookie"]
COOKIES_LOGIN = config_douyin["TokenManager"]["douyin"]["headers"]["Cookie-Login"]

COOKIES_LOCK = threading.Lock()
UPDATE_COOKIES_TIME = 90


TTWID = ""
TTWID_LOCK = threading.Lock()


def gen_ttwid_login() -> str:
    global TTWID
    douyin_logger.info("更新ttwid_login中 ......")
    retries = 0
    max_retries = 3
    while retries < 3:
        proxy = get_random_proxy()
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36",
                "Cookie": f"__ac_signature=_02B4Z6wo00f01XEYf2gAAIDAWBzhFg5L96FxKHvAADQSa4; ttwid={TTWID}; enter_pc_once=1; UIFID_TEMP=750f6e51337de6ac00110e67d7b208d4484528b0becad6c584ba6ece6ce7703a5752f8e39b0ba6e1079462b867f4c7dda37c6f1736fb483f0eb8cd7907f999e6048b1c082e24cdfcc8c78ead98a7aef6; s_v_web_id=verify_mbrche3p_qLtYKnUZ_Gxhu_4LLS_9DfV_fhhNF4yP1Sd9; hevc_supported=true; fpk1=U2FsdGVkX19l6RwXw/CiFAvWqJWCNKbLatj1zGqNVhkR8EJu9NcMEZD83gMJf2l6V1XAJgbcktIjuyHf9prVdg==; fpk2=94176c67796dfc38822d13202360e8de; strategyABtestKey=%221749609769.881%22; passport_csrf_token=ca879ed1c9a288b177ae0ba7bd617127; passport_csrf_token_default=ca879ed1c9a288b177ae0ba7bd617127; __security_mc_1_s_sdk_crypt_sdk=a97e4602-4fdf-8746; __security_mc_1_s_sdk_cert_key=82c55d98-40c5-9fe2; bd_ticket_guard_client_web_domain=2; is_dash_user=1; volume_info=%7B%22isUserMute%22%3Afalse%2C%22isMute%22%3Atrue%2C%22volume%22%3A0.875%7D; UIFID=750f6e51337de6ac00110e67d7b208d4484528b0becad6c584ba6ece6ce7703a5752f8e39b0ba6e1079462b867f4c7ddcc46d844b7d1812befce31cbe30c9a2c5592335f66f2930588324db296b6e392351a4450614b7994893ea73f40c18804b3b255689806fef9dc3e5d79e2fdcff5c52b8f7e807a488e49b06edf76d89d7cffba1e7111e9516548eed8f7c6f8354fad64062000210eff86db346fc1d551f0; douyin.com; device_web_cpu_core=8; device_web_memory_size=8; xgplayer_user_id=519154218589; xg_device_score=7.495228914558955; stream_player_status_params=%22%7B%5C%22is_auto_play%5C%22%3A0%2C%5C%22is_full_screen%5C%22%3A0%2C%5C%22is_full_webscreen%5C%22%3A0%2C%5C%22is_mute%5C%22%3A1%2C%5C%22is_speed%5C%22%3A1%2C%5C%22is_visible%5C%22%3A0%7D%22; download_guide=%223%2F20250611%2F0%22; __ac_nonce=068492284006d2691bd55; gulu_source_res=eyJwX2luIjoiMWNkZWMwMzk4N2I2YTY3MTBmNjVkZjViMzdhYzFkZWRkZmIzZmE2ZDBjM2ZkZjlhM2UzMjVlYWZmZDQwYTIxOSJ9; FORCE_LOGIN=%7B%22videoConsumedRemainSeconds%22%3A180%2C%22isForcePopClose%22%3A1%7D; biz_trace_id=52d983a8; sdk_source_info=7e276470716a68645a606960273f276364697660272927676c715a6d6069756077273f276364697660272927666d776a68605a607d71606b766c6a6b5a7666776c7571273f275e58272927666a6b766a69605a696c6061273f27636469766027292762696a6764695a7364776c6467696076273f275e582729277672715a646971273f2763646976602729277f6b5a666475273f2763646976602729276d6a6e5a6b6a716c273f2763646976602729276c6b6f5a7f6367273f27636469766027292771273f273030343135323637333c313234272927676c715a75776a716a666a69273f2763646976602778; bit_env=NpDeYrPRlXe_1pSWQpDN3ePy8vp21Ib-uhcq7__HuzTIgGI2UDjQY7c5ugqW_Hj85OD3k5VBuiAqyYJDz6Zw_PDSOTwrmWsW1x-_idlYlfVYR6agAI78laIBWxAAceWXmXt3gB2yv0mh6pjMyPVywJEeLmd6NMiy-PCVUAfU7odqWPp9ECgslQUL4y3WEGcHMtgYzxjD_qgpps5Zf7H-1C_Wnvarj5CTopokXfGPfL-1y_uWXYdNBiFcOdQCMVQd6K3Bf9o-jDgqpQf13QnwSr4kQLrq0942UpivpPuQ-jqFPSmZwsPLbwiw1eGrZvYcdJO5K-cztFW_I9BypoSrLuzSVDuVpLKxGe-kd4Ay7ANA_BBWUD9smbvtdR5nrAeG58loTeJyxhfWG-V8NLRw-cmEco5_D_B3bLf9sfoHlDXky6sa7PJ_NuKNWNg3nMiQehBH4oVw5GFrEw1xE_KSHMgGaBTvb9OXCFhMr_EEGu8m2FM5kYewaweCJQAnXZCYX_LZjxBuk3wsXtP4vYxjPxQddLPEsuRz__gFyOdvg5mT88LJJH4F7c5JtSwGVyUy; passport_assist_user=CkH2jyj7SHi-QTAJmWi4jUNtMEMe9tyaZ_f4pTerwLImLUYO4biax8mSfi4Gn9vUIxqDT7jGvQFAZRDQPLjJ-4GPQhpKCjwAAAAAAAAAAAAATxqZ_B43mBJMrzpKNFTXpkHZOH-Tf-9ZZS-SUBnduq1fis0Vxhjay71zD_Z1pcTMCNgQ--XzDRiJr9ZUIAEiAQMi38VD; n_mh=gwpzYVrBdls7OMUnSxJJZsA6_ublnzn4n530AV7Tr00; sid_guard=1d3df6ff0769e9c246fbb9b2a4dcfdbc%7C1749623746%7C5184000%7CSun%2C+10-Aug-2025+06%3A35%3A46+GMT; uid_tt=045eaf6c34a1bc4c78195a9cf7166004; uid_tt_ss=045eaf6c34a1bc4c78195a9cf7166004; sid_tt=1d3df6ff0769e9c246fbb9b2a4dcfdbc; sessionid=1d3df6ff0769e9c246fbb9b2a4dcfdbc; sessionid_ss=1d3df6ff0769e9c246fbb9b2a4dcfdbc; is_staff_user=false; sid_ucp_v1=1.0.0-KDlkMWQxMmY3OWJmZDQ2MzUyZWVlMWVjYWY5NjlhYzMyOTBlMGVlMzIKIQiA_uDq4M2AAxDCx6TCBhjvMSAMMLyqg6cGOAdA9AdIBBoCbHEiIDFkM2RmNmZmMDc2OWU5YzI0NmZiYjliMmE0ZGNmZGJj; ssid_ucp_v1=1.0.0-KDlkMWQxMmY3OWJmZDQ2MzUyZWVlMWVjYWY5NjlhYzMyOTBlMGVlMzIKIQiA_uDq4M2AAxDCx6TCBhjvMSAMMLyqg6cGOAdA9AdIBBoCbHEiIDFkM2RmNmZmMDc2OWU5YzI0NmZiYjliMmE0ZGNmZGJj; login_time=1749623746141; SelfTabRedDotControl=%5B%5D; FOLLOW_NUMBER_YELLOW_POINT_INFO=%22MS4wLjABAAAA80u-K6ZCISqz6vVX0_d7rhDM0qCkHiidkrfAF8F1bAAgPVM85ML1CvB-5FA_HU88%2F1749657600000%2F0%2F1749623747363%2F0%22; _bd_ticket_crypt_cookie=6654c1dfc9c543f030ede540babf29e3; __security_mc_1_s_sdk_sign_data_key_web_protect=2954f1d5-40e9-abf2; __security_server_data_status=1; publish_badge_show_info=%220%2C0%2C0%2C1749623752119%22; dy_swidth=1024; dy_sheight=1366; IsDouyinActive=true; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1024%2C%5C%22screen_height%5C%22%3A1366%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A8%2C%5C%22device_memory%5C%22%3A8%2C%5C%22downlink%5C%22%3A10%2C%5C%22effective_type%5C%22%3A%5C%224g%5C%22%2C%5C%22round_trip_time%5C%22%3A100%7D%22; home_can_add_dy_2_desktop=%221%22; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCTW5CMUV1Z05EMEticm1UaUNBdWczOW9FU3dPQ3RIU2pCRzJuWnlDTVlJNGpESS9tQnFMcGRCa0JKMVFjeXpTdldHTTdGVDlUUVlhRTNVQUJEWEJqZ1E9IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoyfQ%3D%3D; passport_fe_beating_status=true; odin_tt=58ae809c449b8663ac80a52ecf5d5b84d9f1b2bc500c33df06d11bd71af46824a991160889321a5b5de604b688cd2b465046384e92cf8b3e1bec07f62e39be3e"
            }
            url = config_douyin["TokenManager"]["douyin"]["ttwid"]["url"]
            data = config_douyin["TokenManager"]["douyin"]["ttwid"]["data"]
            response = requests.post(url, headers=headers, data=data ,proxies=proxies)
            with TTWID_LOCK:
                TTWID = response.cookies["ttwid"]
            douyin_logger.info(f"获取ttwid_login成功: {TTWID}")
            feishu_api.update_douyin_ttwid(access_token,app_token, sheet_keyword_list_douyin,TTWID)
            return TTWID

        except Exception as e:
            douyin_logger.error(f"获取ttwid_login时出错: {e}")
            retries += 1
            time.sleep(5)
            continue

def gen_ttwid() -> str:
    global TTWID
    douyin_logger.info("更新ttwid中 ......")
    retries = 0
    max_retries = 3
    while retries < 3:
        proxy = get_random_proxy()
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            headers = {
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36",
            }
            url = config_douyin["TokenManager"]["douyin"]["ttwid"]["url"]
            data = config_douyin["TokenManager"]["douyin"]["ttwid"]["data"]
            response = requests.post(url, headers=headers, data=data ,proxies=proxies)
            with TTWID_LOCK:
                TTWID = response.cookies["ttwid"]
            douyin_logger.info(f"获取ttwid成功: {TTWID}")
            return TTWID

        except Exception as e:
            douyin_logger.error(f"获取ttwid时出错: {e}")
            retries += 1
            time.sleep(5)
            continue

def get_douyin_user_info_v2(sec_uid):
    retries = 0
    max_retries = 3
    while retries < max_retries:
        proxy = get_random_proxy()
        douyin_logger.info(f"Attempt {retries + 1}/{max_retries} with proxy: {proxy}")
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            kwargs = get_douyin_headers()
            headers = kwargs["headers"]
            # headers["User-Agent"] = "Mozilla/5.0 (Linux; Android 13; SM-G981B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36"

            params = DEFAULT_PARAMS.copy()
            params["sec_user_id"] = sec_uid
            endpoint = BogusManager.xb_model_2_endpoint("https://www.douyin.com/aweme/v1/web/user/profile/other/",params,kwargs["headers"]["User-Agent"])
            headers["Cookie"] = COOKIES_LOGIN + "; ttwid=" + TTWID

            # headers["Cookie"] = "ttwid=1%7ClPeq6t5UxjR4_B9czB03RHtqO9UUqaJ0SjP9ThIWJdw%7C1749522040%7C4f1846f9e29d04249c312cda047d31eace74c7c53055d52e789ba2d1024e9d49; volume_info=%7B%22volume%22%3A0.6%2C%22isMute%22%3Afalse%7D; is_dash_user=1; FORCE_LOGIN=%7B%22videoConsumedRemainSeconds%22%3A180%2C%22isForcePopClose%22%3A1%7D; IsDouyinActive=true; hevc_supported=true; home_can_add_dy_2_desktop=%220%22; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1470%2C%5C%22screen_height%5C%22%3A956%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A8%2C%5C%22device_memory%5C%22%3A0%2C%5C%22downlink%5C%22%3A%5C%22%5C%22%2C%5C%22effective_type%5C%22%3A%5C%22%5C%22%2C%5C%22round_trip_time%5C%22%3A0%7D%22; dy_sheight=956; dy_swidth=1470; enter_pc_once=1; biz_trace_id=2868104a; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCR1RGb1lLT0s4bnMwL0JLNUo1NEc0Z0EwMzRJRVk3UHNpMG03ZnJMZENhSUh0V1loMW9EdHdGQ1orK01Fak84MFhEMG1hRE9kcVpmL1hFRTFmMHZCMFU9IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoyfQ%3D%3D; bd_ticket_guard_client_web_domain=2; stream_player_status_params=%22%7B%5C%22is_auto_play%5C%22%3A0%2C%5C%22is_full_screen%5C%22%3A0%2C%5C%22is_full_webscreen%5C%22%3A1%2C%5C%22is_mute%5C%22%3A0%2C%5C%22is_speed%5C%22%3A1%2C%5C%22is_visible%5C%22%3A0%7D%22; device_web_memory_size=-1; douyin.com=; __security_mc_1_s_sdk_cert_key=280a9261-4247-865b; __security_mc_1_s_sdk_crypt_sdk=205b109a-4955-8e8a; __security_mc_1_s_sdk_sign_data_key_web_protect=a4eab462-4967-928f; __ac_nonce=06847e688007a2a151d22; __ac_signature=_02B4Z6wo00f01p1k68wAAIDB-375pR0EqIqdVO9AAM8r09; xg_device_score=7.6; download_guide=%223%2F20250610%2F1%22; passport_csrf_token=9510b73fcaff44954d3b91d5f32983bf; passport_csrf_token_default=9510b73fcaff44954d3b91d5f32983bf; strategyABtestKey=%221749523184.362%22; s_v_web_id=verify_mbpwa1ky_x83p7Yx5_gFRR_4n9v_Awly_KIjhRqbOQaE0; WallpaperGuide=%7B%22showTime%22%3A1749095892392%2C%22closeTime%22%3A0%2C%22showCount%22%3A1%2C%22cursor1%22%3A10%2C%22cursor2%22%3A2%7D; device_web_cpu_core=8; odin_tt=178567d8cf94f4de7bbb23f198f7d7eae8c40bb110445fc0e17391ded5711217adf6c89711a0ddba097b32fe48da2e1060d635b8d7761b0de2bb203d4ce9cfce8f6d81544adfa0db3488c0bc24f5597b; __security_mc_1_s_sdk_sign_data_key_sso=797dcba1-49d1-930a; SEARCH_RESULT_LIST_TYPE=%22multi%22; SearchMultiColumnLandingAbVer=2; xgplayer_user_id=364958318589; UIFID=0dc278b5ee1a9490bcb11e245f69d82a6711f73c24592a7abb7a84f4dbcd6cc1c159d92a5ccb0d43900cb2e72ff070be3df056d46481946ba59ae894b3b760b89d348bb70d28c5e6cbf9b15408057ba76495102bb02976a6a45c1ec25de95672888f7bff061514015417864c4f9eb4b9b71a56c4e781f66c14c4f186b1846fd577d3bd23e7c64a1c2b2b471341ef0e2e003b1b5ac3dad0566b837286e98debd1; fpk1=U2FsdGVkX1/MGDZkkbI9zhoo5P2I0uhRISMuJ4h6dCTfAbvR6hE2vSZURJF28vdKVHd/BZgQekGUbCExd1BxEA==; fpk2=43544f0c81a090558dc27390694769b1; UIFID_TEMP=0dc278b5ee1a9490bcb11e245f69d82a6711f73c24592a7abb7a84f4dbcd6cc1791064fa0e0219e30c84395f6ce851f02ae9c31abccb39dbfaca38e8e22c712338bc7f515ed23fec69db9887e217b0f1"

            response = requests.get(endpoint, headers=headers, proxies=proxies,timeout=4)
            douyin_logger.info(f"Response status code: {response.status_code}")
            response.raise_for_status()
            data = response.json() 
            user_info = {
                "sec_uid": sec_uid,
                "aweme_count": data["user"]["aweme_count"],
                "nickname": data["user"]["nickname"],
                "origin_id": data["user"]["unique_id"],
                "gender": 0,
                "ip_location": "",
                "follower_count": data["user"]["mplatform_followers_count"],
                "register_time": get_register_time_by_sec_uid(sec_uid),
                "avatar_url": data["user"]["avatar_300x300"]["url_list"][0],
                "signature": data["user"]["signature"],
                "home_page_url": f"https://www.douyin.com/user/{sec_uid}",
                "seasons_series": str(get_series_by_sec_uid(sec_uid)),
            }

            if "ip_location" in data["user"]:
                user_info["ip_location"] = parse_ip_location(data["user"]["ip_location"])

            print(user_info)
            return user_info
        except (ProxyError, ConnectionError, Timeout, SSLError) as e:
            douyin_logger.warning(f"Proxy connection failed ({type(e).__name__})")
            add_proxy_to_blacklist(proxy["ip"])
            retries += 1

        except Exception as e:
            douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
            print(response.json())
            retries += 1
            break
    return {}

def get_register_time_by_sec_uid(sec_uid):
    max_retries = 3
    retries = 0
    while retries < max_retries:
        proxy = get_random_proxy()
        douyin_logger.info(f"Attempt {retries + 1}/{max_retries} with proxy: {proxy}")
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            kwargs = get_douyin_headers()
            headers = kwargs["headers"]
            url = "https://www.douyin.com/aweme/v1/web/aweme/post"
            params = DEFAULT_PARAMS.copy()
            params["sec_user_id"] = sec_uid
            params["max_cursor"] = 0
            params["count"] = 18
            params["forward_anchor_cursor"] = '1362275200000' 
            params["forward_end_cursor"] = str(int(datetime.now().timestamp() * 1000))
            params["publish_video_strategy_type"] = "2"
            params["msToken"] = ''
            a_bogus = gen_a_bogus(params, kwargs["headers"]["User-Agent"])
            headers["Cookie"] = COOKIES_LOGIN + "; ttwid=" + TTWID

            endpoint = f"https://www.douyin.com/aweme/v1/web/aweme/post/?{urlencode(params)}&a_bogus={a_bogus}"
            response = requests.get(endpoint, headers = headers, proxies=proxies)

            douyin_logger.info(f"Response status code: {response.status_code}")
            if len(response.json()["aweme_list"]) > 0:
                for item in reversed(response.json()["aweme_list"]):
                    if item["create_time"]:
                        return time_stamp_to_datetime(item["create_time"])
            return ""
        except (ProxyError, ConnectionError, Timeout, SSLError) as e:
            douyin_logger.warning(f"Proxy connection failed ({type(e).__name__})")
            add_proxy_to_blacklist(proxy["ip"])
            retries += 1

        except Exception as e:
            douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
            retries += 1
    return ""


def time_stamp_to_datetime(time_stamp):
    """
    将时间戳转换为日期时间
    """
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time_stamp))



def parse_ip_location(input_str):
    if not input_str:
        return None
    # 匹配 "IP属地：" 或 "IP属地:" 后的内容，兼容中英文冒号和空格
    match = re.search(r'IP属地[：:]\s*(.*)', input_str)
    return match.group(1).strip() if match else None



def get_video_link(sec_uid,aweme_id):
    return f"https://www.douyin.com/user/{sec_uid}?modal_id={aweme_id}"


def hashtags_to_array(tag_string) -> list:
    return [tag.strip('#').strip() for tag in tag_string.split() if tag.startswith('#')]


def parse_json(response: Response) -> dict:
    """解析JSON响应对象 (Parse JSON response object)
    Args:
        response (Response): 原始响应对象 (Raw response object)

    Returns:
        dict: 解析后的JSON数据 (Parsed JSON data)
    """
    if (
            response is not None
            and isinstance(response, Response)
            and response.status_code == 200
    ):
        try:
            return response.json()
        except json.JSONDecodeError as e:        
            # 尝试使用正则表达式匹配response.text中的json数据
            match = re.search(r"\{.*\}", response.text)
            try:
                return json.loads(match.group())
            except json.JSONDecodeError as e:
                douyin_logger.error("解析 {0} 接口 JSON 失败： {1}".format(response.url, e))
                raise TypeError("解析JSON数据失败")
    else:
        if isinstance(response, Response):
            douyin_logger.error(
                "获取数据失败。状态码: {0}".format(response.status_code)
            )
        else:
            douyin_logger.error("无效响应类型。响应类型: {0}".format(type(response)))

        douyin_logger.error("获取数据失败")
        raise TypeError("解析JSON数据失败")


def get_posts_by_sec_uid(sec_uid):
    video_list = []
    max_cursor = 0
    has_more = 1
    max_retries = 3
    retries = 0
    p = 0
    while has_more == 1 and len(video_list) < 500:
        p += 1
        if p > 25:
            return video_list
        while retries < max_retries:
            proxy = get_random_proxy()
            douyin_logger.info(f"Using proxy: {proxy}")
            try:
                proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
                proxies = {'http': proxy_url, 'https': proxy_url}
                kwargs = get_douyin_headers()
                headers = kwargs["headers"]
                url = "https://www.douyin.com/aweme/v1/web/aweme/post"
                params = DEFAULT_PARAMS.copy()
                params["sec_user_id"] = sec_uid
                params["max_cursor"] = max_cursor
                print(f"max_cursor:{max_cursor}")
                params["count"] = 30
                a_bogus = gen_a_bogus(params, kwargs["headers"]["User-Agent"])
                headers["Cookie"] = COOKIES_LOGIN + "; ttwid=" + TTWID
                
                endpoint = f"https://www.douyin.com/aweme/v1/web/aweme/post/?{urlencode(params)}&a_bogus={a_bogus}"
                
                response = requests.get(endpoint, headers=headers, proxies=proxies,timeout=5)

                result = response.json()

                max_cursor = result["max_cursor"]
                has_more = result["has_more"]
                length = len(result["aweme_list"])
                print(has_more)
                douyin_logger.info(f"{length} videos found")
                for video in result["aweme_list"]:
                    aweme_id = video["aweme_id"]
                    title = video["desc"]
                    like = video["statistics"]["digg_count"]
                    collect = video["statistics"]["collect_count"]
                    share = video["statistics"]["share_count"]
                    comment = video["statistics"]["comment_count"]
                    play_count = video["statistics"]["play_count"]
                    duration = video["duration"]
                    caption = video["caption"]
                    create_time = time_stamp_to_datetime(video["create_time"])
                    cur_post = {
                        "sec_uid": sec_uid,
                        "aweme_id": aweme_id,
                        "title": title,
                        "like": like,
                        "collect": collect,
                        "share": share,
                        "comment": comment,
                        "play_count": play_count,
                        "video_link": get_video_link(sec_uid, aweme_id),
                        "create_time": create_time,
                        "duration": duration//1000,
                        "tags": str(hashtags_to_array(caption))
                    }
                    video_list.append(cur_post)
                break
            except (ProxyError, ConnectionError, Timeout, SSLError) as e:
                douyin_logger.warning(f"Proxy connection failed ({type(e).__name__}): {str(e)}")
                add_proxy_to_blacklist(proxy["ip"])
                retries += 1
                continue
            except Exception as e:
                douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
                retries  += 1
                continue

    return video_list


def get_first_post_by_sec_uid(sec_uid):
    video_list = []
    max_cursor = 0
    has_more = 1
    max_retries = 3
    retries = 0
    while has_more == 1:
        while retries < max_retries:
            proxy = get_random_proxy()
            douyin_logger.info(f"Using proxy: {proxy}")
            try:
                proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
                proxies = {'http': proxy_url, 'https': proxy_url}
                kwargs = get_douyin_headers()
                headers = kwargs["headers"]
                url = "https://www.douyin.com/aweme/v1/web/aweme/post"
                params = DEFAULT_PARAMS.copy()
                params["sec_user_id"] = sec_uid
                params["max_cursor"] = max_cursor
                params["count"] = 18
                params["msToken"] = ''
                a_bogus = gen_a_bogus(params, kwargs["headers"]["User-Agent"])
                print(TTWID)
                headers["Cookie"] = COOKIES_LOGIN + "; ttwid=" + TTWID
                # headers["Cookie"] = "ttwid=1%7ClPeq6t5UxjR4_B9czB03RHtqO9UUqaJ0SjP9ThIWJdw%7C1749522040%7C4f1846f9e29d04249c312cda047d31eace74c7c53055d52e789ba2d1024e9d49; volume_info=%7B%22volume%22%3A0.6%2C%22isMute%22%3Afalse%7D; is_dash_user=1; FORCE_LOGIN=%7B%22videoConsumedRemainSeconds%22%3A180%2C%22isForcePopClose%22%3A1%7D; IsDouyinActive=true; hevc_supported=true; home_can_add_dy_2_desktop=%220%22; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1470%2C%5C%22screen_height%5C%22%3A956%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A8%2C%5C%22device_memory%5C%22%3A0%2C%5C%22downlink%5C%22%3A%5C%22%5C%22%2C%5C%22effective_type%5C%22%3A%5C%22%5C%22%2C%5C%22round_trip_time%5C%22%3A0%7D%22; dy_sheight=956; dy_swidth=1470; enter_pc_once=1; biz_trace_id=2868104a; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCR1RGb1lLT0s4bnMwL0JLNUo1NEc0Z0EwMzRJRVk3UHNpMG03ZnJMZENhSUh0V1loMW9EdHdGQ1orK01Fak84MFhEMG1hRE9kcVpmL1hFRTFmMHZCMFU9IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoyfQ%3D%3D; bd_ticket_guard_client_web_domain=2; stream_player_status_params=%22%7B%5C%22is_auto_play%5C%22%3A0%2C%5C%22is_full_screen%5C%22%3A0%2C%5C%22is_full_webscreen%5C%22%3A1%2C%5C%22is_mute%5C%22%3A0%2C%5C%22is_speed%5C%22%3A1%2C%5C%22is_visible%5C%22%3A0%7D%22; device_web_memory_size=-1; douyin.com=; __security_mc_1_s_sdk_cert_key=280a9261-4247-865b; __security_mc_1_s_sdk_crypt_sdk=205b109a-4955-8e8a; __security_mc_1_s_sdk_sign_data_key_web_protect=a4eab462-4967-928f; __ac_nonce=06847e688007a2a151d22; __ac_signature=_02B4Z6wo00f01p1k68wAAIDB-375pR0EqIqdVO9AAM8r09; xg_device_score=7.6; download_guide=%223%2F20250610%2F1%22; passport_csrf_token=9510b73fcaff44954d3b91d5f32983bf; passport_csrf_token_default=9510b73fcaff44954d3b91d5f32983bf; strategyABtestKey=%221749523184.362%22; s_v_web_id=verify_mbpwa1ky_x83p7Yx5_gFRR_4n9v_Awly_KIjhRqbOQaE0; WallpaperGuide=%7B%22showTime%22%3A1749095892392%2C%22closeTime%22%3A0%2C%22showCount%22%3A1%2C%22cursor1%22%3A10%2C%22cursor2%22%3A2%7D; device_web_cpu_core=8; odin_tt=178567d8cf94f4de7bbb23f198f7d7eae8c40bb110445fc0e17391ded5711217adf6c89711a0ddba097b32fe48da2e1060d635b8d7761b0de2bb203d4ce9cfce8f6d81544adfa0db3488c0bc24f5597b; __security_mc_1_s_sdk_sign_data_key_sso=797dcba1-49d1-930a; SEARCH_RESULT_LIST_TYPE=%22multi%22; SearchMultiColumnLandingAbVer=2; xgplayer_user_id=364958318589; UIFID=0dc278b5ee1a9490bcb11e245f69d82a6711f73c24592a7abb7a84f4dbcd6cc1c159d92a5ccb0d43900cb2e72ff070be3df056d46481946ba59ae894b3b760b89d348bb70d28c5e6cbf9b15408057ba76495102bb02976a6a45c1ec25de95672888f7bff061514015417864c4f9eb4b9b71a56c4e781f66c14c4f186b1846fd577d3bd23e7c64a1c2b2b471341ef0e2e003b1b5ac3dad0566b837286e98debd1; fpk1=U2FsdGVkX1/MGDZkkbI9zhoo5P2I0uhRISMuJ4h6dCTfAbvR6hE2vSZURJF28vdKVHd/BZgQekGUbCExd1BxEA==; fpk2=43544f0c81a090558dc27390694769b1; UIFID_TEMP=0dc278b5ee1a9490bcb11e245f69d82a6711f73c24592a7abb7a84f4dbcd6cc1791064fa0e0219e30c84395f6ce851f02ae9c31abccb39dbfaca38e8e22c712338bc7f515ed23fec69db9887e217b0f1"
                # headers["Cookie"] = "__ac_nonce=06847e30600356cdb7e9b; __ac_signature=_02B4Z6wo00f010zfgzQAAIDCZdsdSi2lr4NM74eAALt799; ttwid=1%7ClPeq6t5UxjR4_B9czB03RHtqO9UUqaJ0SjP9ThIWJdw%7C1744110068%7C02496a68f91a95c90bb593c5c0b001ad2909f99ca1affe31196f1e0153da6ec5; enter_pc_once=1; UIFID_TEMP=750f6e51337de6ac00110e67d7b208d4484528b0becad6c584ba6ece6ce7703a5752f8e39b0ba6e1079462b867f4c7dd9bc6b37d972a6f1f3f63a9467bfcf4556d05dba7cee91f7935dd15886ae1787c; hevc_supported=true; dy_swidth=1024; dy_sheight=1366; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1024%2C%5C%22screen_height%5C%22%3A1366%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A8%2C%5C%22device_memory%5C%22%3A8%2C%5C%22downlink%5C%22%3A10%2C%5C%22effective_type%5C%22%3A%5C%224g%5C%22%2C%5C%22round_trip_time%5C%22%3A250%7D%22; fpk1=U2FsdGVkX1+Pv/C3urO1y9PoRX0vRNpkNjcT0kI49PbJDj7Z2xuJIDDY0afLeIBxdRPrS58ZzJajCiR7omkznQ==; fpk2=94176c67796dfc38822d13202360e8de; strategyABtestKey=%221749541639.56%22; passport_csrf_token=eca164d20bdf6f2ddb0eb44f5c8a7a68; passport_csrf_token_default=eca164d20bdf6f2ddb0eb44f5c8a7a68; s_v_web_id=verify_mbq7x54o_HvVrjUr2_5eO0_49mk_9Njd_t5p2Yagr1j2s; biz_trace_id=bb515346; __security_mc_1_s_sdk_crypt_sdk=7e195e7f-471b-9727; __security_mc_1_s_sdk_cert_key=1f4837df-462d-a090; __security_mc_1_s_sdk_sign_data_key_web_protect=0a92736e-4f77-a745; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCTW5CMUV1Z05EMEticm1UaUNBdWczOW9FU3dPQ3RIU2pCRzJuWnlDTVlJNGpESS9tQnFMcGRCa0JKMVFjeXpTdldHTTdGVDlUUVlhRTNVQUJEWEJqZ1E9IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoyfQ%3D%3D; bd_ticket_guard_client_web_domain=2; sdk_source_info=7e276470716a68645a606960273f276364697660272927676c715a6d6069756077273f276364697660272927666d776a68605a607d71606b766c6a6b5a7666776c7571273f275e58272927666a6b766a69605a696c6061273f27636469766027292762696a6764695a7364776c6467696076273f275e582729277672715a646971273f2763646976602729277f6b5a666475273f2763646976602729276d6a6e5a6b6a716c273f2763646976602729276c6b6f5a7f6367273f27636469766027292771273f273637363531333431303c313234272927676c715a75776a716a666a69273f2763646976602778; bit_env=SQb0gWaXudo0ciR25ZWebrVXrx5PSy0WhKWLr-YXEU6bfVJWmcgvVmApKgMjHrxhSaMGHvPujIvZoZEsj9uQ7B3NmGCVBWj1u8nWtmwCgP1-dFznQ7Nd4OTGvTu8vfqNuMXrTBqBA7T9VmzyJxVT0YtPb_mMS6U_d8b3OKexKmBoXwcDIzS2frla-t4yxs8yOfvYP9ILHhJq-3Aerl40GPeDPYg1DKv64cMuxvjRh2aukhi0_S6w7fGboZhQ2tvbvYSb9U0y2d0R4t-KR98FA2VTVg_8qj4LiQdz8AmCjSq2-dof-q08DnYD_f6-ftH-QYtsWfTHfAn76tcsPlRs_73onYuLLLXiYJujF8uagIWfkLbvie-cy8Y9UAX7mp65bZWZJo5J1zaKNRJgxOfK1HRa7Gk1NDIqNeC3HABUzFE71FhRmM4r0mO5dIYESswBf1HIOILDgdNEI3jpeqMXivMlo-FMHDLmDM1UnNnzpE0NasBBD5VP3mnboMgUoR9teYABeEsXir81ppTBcCOy4NaFMMYIV5CXI3ow9mCRD2EpNMC717QB17BckSKAxLKN; gulu_source_res=eyJwX2luIjoiMGZmZDg4YzU0Y2ZiMmU4OWQxY2Y3M2VhMTFlMDUwNGM1NDFmYmNhNjVjOTdlM2Q5OGNjZGEwMzA0YmZkMDg4ZSJ9; passport_auth_mix_state=wnf73ej6zcmum8ispgpa6d43m8jhgp6fuzb8srbn4972rz0k; odin_tt=eaceea35954871f2cb5fbdf7b1ef70ee23b8762e70c59acf7f2b76c5b5b85c224388cfe5d98b3f192baf87844d654442b1635ea0a5a3ecba10d57235dc05f73be691e6c47046b90b39e8dd78a7b28e86; FORCE_LOGIN=%7B%22videoConsumedRemainSeconds%22%3A180%7D; download_guide=%221%2F20250610%2F0%22; UIFID=750f6e51337de6ac00110e67d7b208d4484528b0becad6c584ba6ece6ce7703a87b8a934b4526bf0ca7281c6ab414d6db61d11927e4bb63738a78f6ee1f28384c92040918fafc79972a946e1f86c3f2ca29740b37d9e44e4bc3e2afabb896c3359af5284d148788744557d90e54653f7ebc0556b29beb00eb72b5bc8b01463b3c9bcf5efb87f1948850d4614c637e4fa48f0792d413096b7743297fd7af6fca4; IsDouyinActive=true; home_can_add_dy_2_desktop=%220%22"
                # headers["Cookie"] = "ttwid=1%7ClPeq6t5UxjR4_B9czB03RHtqO9UUqaJ0SjP9ThIWJdw%7C1744110068%7C02496a68f91a95c90bb593c5c0b001ad2909f99ca1affe31196f1e0153da6ec5; UIFID_TEMP=1092a36a2acc3ae8398deab3fd09104051338a3ad6cb249c7479b2d1664bb2d4631aba278b1aaac3eba7d154f7db37fe7418e5330cfbf1d293037167188872bd4494ac506718ff7797a6a8f8fabf2538; hevc_supported=true; fpk1=U2FsdGVkX19QjAPhV6WrFFIILgDtego2icH0tORN+owM9xhJKviNbkHr/KJqPITQ3JJB56r1fiNYQg5YTKuR2w==; fpk2=4ad072dfe5c14cff71c013c0dea8a6a3; odin_tt=801a1377d9f20465f88c2d74854fe15e24682e61cbbb2260bf922a2f5e8ef4e1d8e8c1d1612414a11117124847212921e0a002c5547bfc511358133522dee7c91d49933ef13c2a79c4839ebe60dcb35c; bd_ticket_guard_client_web_domain=2; UIFID=1092a36a2acc3ae8398deab3fd09104051338a3ad6cb249c7479b2d1664bb2d4631aba278b1aaac3eba7d154f7db37febbd800a49273577b17063855bba3750b564b0b819ea112ecf6028e3484811ab33984626fec9977801e70864b41455ae3353f450a5f9729325f210ebaec6c0713dfe8ff7d2dd4b3d5151c444e04b8e5d6e1722e5b84e7c7da57df080d91ef189350cc0e2ea5620259b279d9eddfa058cf; SearchMultiColumnLandingAbVer=2; SEARCH_RESULT_LIST_TYPE=%22multi%22; volume_info=%7B%22isUserMute%22%3Atrue%2C%22isMute%22%3Atrue%2C%22volume%22%3A1%7D; xgplayer_device_id=82563201101; xgplayer_user_id=315237500452; FORCE_LOGIN=%7B%22videoConsumedRemainSeconds%22%3A180%7D; is_dash_user=1; douyin.com; xg_device_score=7.568284552297575; device_web_cpu_core=8; device_web_memory_size=8; download_guide=%223%2F20250605%2F0%22; enter_pc_once=1; s_v_web_id=verify_mbopilj3_ljS9tOsm_k54h_49cL_9HWf_ujM3Uomfpxz4; passport_csrf_token=ba467cac9841be86eea9536be14303ba; passport_csrf_token_default=ba467cac9841be86eea9536be14303ba; __security_mc_1_s_sdk_crypt_sdk=557be796-4f3b-b35f; __security_mc_1_s_sdk_cert_key=b29a8ec1-4f1f-9993; __security_mc_1_s_sdk_sign_data_key_web_protect=ce255e5e-49f4-b003; __ac_nonce=068479669006f161898d0; __ac_signature=_02B4Z6wo00f01fFxEJAAAIDA2HWO7KmYorXxURQAABQdd4; strategyABtestKey=%221749522026.426%22; biz_trace_id=384e9cf1; bd_ticket_guard_client_data=eyJiZC10aWNrZXQtZ3VhcmQtdmVyc2lvbiI6MiwiYmQtdGlja2V0LWd1YXJkLWl0ZXJhdGlvbi12ZXJzaW9uIjoxLCJiZC10aWNrZXQtZ3VhcmQtcmVlLXB1YmxpYy1rZXkiOiJCTW5CMUV1Z05EMEticm1UaUNBdWczOW9FU3dPQ3RIU2pCRzJuWnlDTVlJNGpESS9tQnFMcGRCa0JKMVFjeXpTdldHTTdGVDlUUVlhRTNVQUJEWEJqZ1E9IiwiYmQtdGlja2V0LWd1YXJkLXdlYi12ZXJzaW9uIjoyfQ%3D%3D; sdk_source_info=7e276470716a68645a606960273f276364697660272927676c715a6d6069756077273f276364697660272927666d776a68605a607d71606b766c6a6b5a7666776c7571273f275e58272927666a6b766a69605a696c6061273f27636469766027292762696a6764695a7364776c6467696076273f275e582729277672715a646971273f2763646976602729277f6b5a666475273f2763646976602729276d6a6e5a6b6a716c273f2763646976602729276c6b6f5a7f6367273f27636469766027292771273f27373d343237353737303c313234272927676c715a75776a716a666a69273f2763646976602778; bit_env=ly2047OqqdUeIZSwjV_rqcuhDVyoQE-rRLVlfHYikjBLi0wXGlt43J8z_rabENiOX9M78zaX8aL6KRLUIJXnpFf8HCBwKBpHmZ9vECNjtDmA9rbU8kPanDEFzdCXnt4TI2NCmI5wKXMqZLNXaQfM14L07nEqQf5WA4xDkh4hfSOs5atRvZkuLXUdCLTBwkXERc0dep7_IxIm4wA8-wPm7KFxkmIkRJJYo73fhopDXLQ0nA2FPCCXxge7HFq3xgU95S_Tr7LqPpCSLgeqCU44yGWBmawJEwGonEqsNMB2pHsfpKeda9DS6zCqcM4PRtilJJyWuyfDMCnhTXBYY2VhgmyzuzOYlcflr9z9goW2Y_d1ughCyItyE0YsN0BQWv_s8RM987N9Ahfsld0yrwoMDyKBlqYjUBErdI-NCfhMcIi-jmll1eTDvnYW8NIOXB0beBPHYxtvVDVSGaWb8rS7ZX3ZF3ynwExWQ4TntfWVAaZg2tt1uKPuEX1JgRho9w9jekeLR0VYLwt3p9wsmbC5J6rpWCTpkJtuin-lOHBh-X3hjT-huIjdgPKjDmq0QP4q; gulu_source_res=eyJwX2luIjoiMWNkZWMwMzk4N2I2YTY3MTBmNjVkZjViMzdhYzFkZWRkZmIzZmE2ZDBjM2ZkZjlhM2UzMjVlYWZmZDQwYTIxOSJ9; passport_auth_mix_state=8akqj6pyfsmltqw0oks09fbb9ltrn0sg; IsDouyinActive=true; home_can_add_dy_2_desktop=%220%22; dy_swidth=1024; dy_sheight=1366; stream_recommend_feed_params=%22%7B%5C%22cookie_enabled%5C%22%3Atrue%2C%5C%22screen_width%5C%22%3A1024%2C%5C%22screen_height%5C%22%3A1366%2C%5C%22browser_online%5C%22%3Atrue%2C%5C%22cpu_core_num%5C%22%3A8%2C%5C%22device_memory%5C%22%3A8%2C%5C%22downlink%5C%22%3A10%2C%5C%22effective_type%5C%22%3A%5C%224g%5C%22%2C%5C%22round_trip_time%5C%22%3A50%7D%22"
                endpoint = f"https://www.douyin.com/aweme/v1/web/aweme/post/?{urlencode(params)}&a_bogus={a_bogus}"
                response = requests.get(endpoint, headers = headers, proxies=proxies,timeout=4)
                result = response.json()
                max_cursor = result["max_cursor"]
                has_more = result["has_more"]
                print(has_more)
                length = len(result["aweme_list"])
                print(f"{length} videos found")
                for video in result["aweme_list"]:
                    aweme_id = video["aweme_id"]
                    title = video["desc"]
                    like = video["statistics"]["digg_count"]
                    collect = video["statistics"]["collect_count"]
                    share = video["statistics"]["share_count"]
                    comment = video["statistics"]["comment_count"]
                    play_count = video["statistics"]["play_count"]
                    create_time = time_stamp_to_datetime(video["create_time"])
                    cur_post = {
                        "sec_uid": sec_uid,
                        "aweme_id": aweme_id,
                        "title": title,
                        "like": like,
                        "collect": collect,
                        "share": share,
                        "comment": comment,
                        "play_count": play_count,
                        "video_link": get_video_link(sec_uid, aweme_id),
                        "create_time": create_time
                    }
                    video_list.append(cur_post)
                return video_list
            except (ProxyError, ConnectionError, Timeout, SSLError) as e:
                douyin_logger.warning(f"Proxy connection failed ({type(e).__name__})")
                add_proxy_to_blacklist(proxy["ip"])
                retries += 1
            except Exception as e:
                douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
                time.sleep(5)
                retries  += 1
        return []
    return []

THIS_UPDATE_CREATORS_DY = []
THIS_UPDATE_KEYWORD_NUM_DY = 0
THIS_UPDATE_CREATOR_NUM_DY = 0
THIS_UPDATE_VIDEO_NUM_DY = 0
THIS_UPDATE_KEYWORDS = []
DETAIL_DATA = []
DATA_LOCK = threading.Lock()
INSERT_LOCK = threading.Lock()
def thread_insert_to_trino():
    while not STOP_THREAD:
        with DATA_LOCK:
            detail = DETAIL_DATA.copy()
            DETAIL_DATA.clear()
        with INSERT_LOCK:
            for item in detail:
                try:
                    with INSERT_TRINO_LOCK:
                        douyin_logger.info(f"第{item['index']}个作者 正在上传数据库......")
                        user_detail = item["user_detail"]
                        video_details = item["video_details"]

                        if user_detail == {}:
                            continue
                        douyin_logger.info(f"第{item['index']}个作者 正在上传用户详细......")
                        insert_douyin_user_detail(user_detail)

                        if video_details == []:
                            continue
                        douyin_logger.info(f"第{item['index']}个作者 正在上传视频详细......")
                        insert_douyin_video_details(video_details, user_detail)
                        douyin_logger.info(f"第{item['index']}个作者 上传完成")
                except  Exception as e:
                    douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
                    continue



def get_authors_douyin(authors):
    need_author_list = []
    not_need_author_list = []
    creator_id_list = get_creators_by_dt_media("douyin")
    for author in authors:
        if str(author["sec_uid"]) in creator_id_list:
            not_need_author_list.append(author)
            continue
        need_author_list.append(author)
    return need_author_list,not_need_author_list

def task_keyword_list_douyin_full_update():
    global THIS_UPDATE_KEYWORD_NUM_DY
    global THIS_UPDATE_KEYWORDS
    start = 2
    end = 100
    keyword_list = feishu_api.get_keyword_list_by_range_douyin(access_token, app_token, sheet_keyword_list_douyin, start, end)
    if len(keyword_list) == 0:
        douyin_logger.info("没有需要查询的关键词")
        return
    while len(keyword_list) > 0:
        print(keyword_list)
        for row in keyword_list:
            keyword = row["keyword"]
            THIS_UPDATE_KEYWORD_NUM_DY += 1
            THIS_UPDATE_KEYWORDS.append(keyword)
            douyin_logger.info(f"正在查询: {keyword}")
            author_list = get_top100_authors_by_keyword(keyword)
            douyin_logger.info(f"查询到 {len(author_list)} 个作者")
            need_author_list,not_need_author_list = get_authors_douyin(author_list)
            douyin_logger.info(f"需要查询 {len(need_author_list)} 个作者")
            print(f"Need:{need_author_list}")
            print(f"Not Need:{not_need_author_list}")
            index = 1
            for author in not_need_author_list:
                sec_uid = author["sec_uid"]
                douyin_logger.info(f"正在更新作者标签{index}/{len(not_need_author_list)}，sec_uid:{sec_uid}")
                index += 1
                tags = get_tags_by_id_and_media(sec_uid, "douyin")
                if keyword in tags:
                    continue
                tags.append(keyword)
                update_tags_by_id_and_media(tags,sec_uid,"douyin")

            index = 1
            for author in need_author_list:
                author['index'] = index
                index = index + 1

            with ThreadPoolExecutor(max_workers=2) as executor:
                futures = [executor.submit(task_author, author,len(need_author_list),keyword) for author in need_author_list]

            start_time = time.time()
            timeout = 20 * 60

            while DETAIL_DATA != [] or INSERT_LOCK.locked():
                current_time = time.time()
                elapsed_time = current_time - start_time
                
                # 检查是否超时
                if elapsed_time > timeout:
                    douyin_logger.warning(f"等待数据库上传超过20分钟，强制结束等待")
                    break
                douyin_logger.info(f"关键词'{keyword}'的{len(need_author_list)}个作者查询完毕，正在上传数据库......")
                time.sleep(5)
            
            DETAIL_DATA.clear()
            if INSERT_LOCK.locked():
                INSERT_LOCK.release()


            douyin_logger.info(f"Keyword : {keyword} Done.")
            feishu_api.update_time_by_rowId(access_token, app_token, sheet_keyword_list_douyin, row["row_id"], THIS_UPDATE_TIME)
        start = end + 1
        end = end + 100
        keyword_list = feishu_api.get_key_list_by_range(access_token, app_token, sheet_keyword_list_douyin, start, end)

def task_author(author : dict,sum,keyword):
    global THIS_UPDATE_KEYWORD_NUM_DY
    global THIS_UPDATE_CREATOR_NUM_DY
    global THIS_UPDATE_VIDEO_NUM_DY
    global THIS_UPDATE_CREATORS_DY
    global DETAIL_DATA
    sec_uid = author["sec_uid"]
    douyin_logger.info(f"正在查询第{author['index']}个作者...... {author['index']}/{sum}")

    if not is_valid_user(sec_uid):
        return {},[]
    THIS_UPDATE_CREATOR_NUM_DY += 1
    THIS_UPDATE_CREATORS_DY.append(sec_uid)

    douyin_logger.info(f"Starting to get user detail,sec_uid:{sec_uid}")
    user_detail = get_douyin_user_info_v2(sec_uid)
    user_detail["tags"] = str([keyword])
    if user_detail == {}:
        douyin_logger.info(f"用户信息查询失败，sec_uid:{sec_uid}")
        return {},[]
    douyin_logger.info(f"用户信息查询完毕，sec_uid:{sec_uid}")

    video_details = get_posts_by_sec_uid(sec_uid)
    if video_details == []:
        douyin_logger.info(f"没有视频，sec_uid:{sec_uid}")
        return {},[]
    douyin_logger.info(f"视频列表查询完毕，sec_uid:{sec_uid}")
    THIS_UPDATE_VIDEO_NUM_DY += len(video_details)

    with DATA_LOCK:
        data_detail = {
            "index" : author['index'],
            "user_detail": user_detail,
            "video_details": video_details
        }
        DETAIL_DATA.append(data_detail)


def get_series_by_sec_uid(sec_uid):
    series = []
    cursor = 0
    max_retries = 3
    p = 0
    while p < 25:
        retries = 0
        while retries < max_retries:
            proxy = get_random_proxy()
            douyin_logger.info(f"Using proxy: {proxy}")
            try:
                proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
                proxies = {'http': proxy_url, 'https': proxy_url}
                kwargs = get_douyin_headers()
                headers = kwargs["headers"]
                url = "https://www.douyin.com/aweme/v1/web/mix/list"
                params = DEFAULT_PARAMS.copy()
                params["sec_user_id"] = sec_uid
                params["cursor"] = cursor
                params["count"] = 6
                headers["Cookie"] = COOKIES
                response = requests.get(url, headers=headers, params=params, timeout=4)
                douyin_logger.info(f"Response status code: {response.status_code}")
                result = response.json()
                cursor = result["cursor"]

                if result["mix_infos"] is None:
                    douyin_logger.info("没有合集")
                    return series

                length = len(result["mix_infos"])
                print(f"{length} series found")

                for item in result["mix_infos"]:
                    series.append(item["mix_name"])

                if result["has_more"] == 1:
                    douyin_logger.info("还有更多合集")
                    break
                else:
                    douyin_logger.info("没有更多合集了")
                    return series
            except (ProxyError, ConnectionError, Timeout, SSLError) as e:
                douyin_logger.warning(f"Proxy connection failed ({type(e).__name__}): {str(e)}")
                add_proxy_to_blacklist(proxy["ip"])
                retries += 1
                continue
            except Exception as e:
                douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
                retries  += 1
                continue
    return series



sheet_keyword_list_bilibili = config.get("bilibili_sheet_keyword_list")
sheet_keyword_list_douyin = config.get("douyin_sheet_keyword_list")
node_token = config.get("node_token")
app_id = config.get("app_id")
app_serect = config.get("app_serect")
access_token = feishu_api.get_aenant_access_token(app_id,app_serect)
app_token = feishu_api.get_app_token(access_token, node_token)
FEISHU_UPDATE_INTERVAL = 5 * 60
FEISHU_LOCK = threading.Lock()

def update_feishu_token():
    douyin_logger.info("Douyin爬虫飞书Token后台更新线程启动")
    global access_token
    global app_token
    while not STOP_THREAD:
        try:
            with FEISHU_LOCK:
                access_token = feishu_api.get_aenant_access_token(app_id, app_serect)
                app_token = feishu_api.get_app_token(access_token, node_token)
            douyin_logger.info("Feishu Token updated")
            time.sleep(FEISHU_UPDATE_INTERVAL)
        except Exception as e:
            douyin_logger.error(f"更新Feishu token时出错: {e}")




oss_manager = OSSManager(get_token())
oss_lock = threading.Lock()
OSS_MANAGER_UPDATE_INTERVAL = 5*60

def update_oss_manager():
    douyin_logger.info("Douyin爬虫Oss后台更新线程启动")
    global oss_manager
    while not STOP_THREAD:
        time.sleep(OSS_MANAGER_UPDATE_INTERVAL)
        if STOP_THREAD:
            break
        try:
            with oss_lock:
                token = get_token()
                oss_manager = OSSManager(token)
            douyin_logger.info("OSS manager updated")
        except Exception as e:
            douyin_logger.error(f"更新OSS manager时出错: {e}")
    douyin_logger.info("OSS manager update thread stopped")


# def update_cookies():
#     global COOKIES
#     while not STOP_THREAD:
#         douyin_logger.info("Update Cookies ...")
#         global COOKIES
#         options = ChromiumOptions()
#         options.headless()
#         options.set_argument('--no-sandbox')
#         options.set_argument('--disable-blink-features=AutomationControlled')
#         dp = ChromiumPage(addr_or_opts=options)
#         dp.get("https://www.douyin.com")
#         runCookies = ""
#         for data in dp.cookies():
#             # if data['name'] == 'verify_fp':
#             #     runCookies += "verify_fp=" + 'verify_maeraygu_TY1R2zpb_g7DC_4hEJ_AgAm_jW6UBP2HTM0G' + "; "
#             #     continue
#             if data['name'] == "ttwid":
#                 ttwid = gen_ttwid()
#                 runCookies += "ttwid=" + ttwid + "; "
#                 continue
#             runCookies += data['name'] + "=" + data['value'] + "; "
#         with COOKIES_LOCK:
#             COOKIES = runCookies
#             print(COOKIES)
#         print(f"Update Cookies Success:{runCookies}")
#         time.sleep(UPDATE_COOKIES_TIME)
#     douyin_logger.info("Update Douyin Cookies Thread Stop")


def update_cookies_v2():
    global COOKIES
    while not STOP_THREAD:
        douyin_logger.info("Update Cookies ...")
        global COOKIES
        runCookies = config_douyin["TokenManager"]["douyin"]["headers"]["Cookie"]
        ttwid = gen_ttwid()
        gen_ttwid_login()
        runCookies += "; ttwid=" + ttwid
        with COOKIES_LOCK:
            COOKIES = runCookies
        time.sleep(UPDATE_COOKIES_TIME)
    douyin_logger.info("Update Douyin Cookies Thread Stop")

STOP_THREAD = False

def thread_update():
    global access_token
    global app_token
    access_token = feishu_api.get_aenant_access_token(app_id,app_serect)
    app_token = feishu_api.get_app_token(access_token, node_token)
    update_douyin_cookies_thread = threading.Thread(target=update_cookies_v2,daemon=True)
    update_douyin_cookies_thread.start()

    oss_manager_update_thread = threading.Thread(target=update_oss_manager, daemon=True)
    oss_manager_update_thread.start()

    feishu_token_update_thread = threading.Thread(target=update_feishu_token, daemon=True)
    feishu_token_update_thread.start()

    thread_insert_to_trino_thread = threading.Thread(target=thread_insert_to_trino, daemon=True)
    thread_insert_to_trino_thread.start()


def get_real_msToken(): 
    """
        生成真实的msToken,当出现错误时返回虚假的值
        (Generate a real msToken and return a false value when an error occurs)
    """
    retries = 0
    while retries < 3:
        proxy = get_random_proxy()
        douyin_logger.info(f"Using proxy: {proxy}")
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            msToken_config = config_douyin["TokenManager"]["douyin"]["msToken"]
            payload = json.dumps(
                {
                    "magic": msToken_config["magic"],
                    "version": msToken_config["version"],
                    "dataType": msToken_config["dataType"],
                    "strData": msToken_config["strData"],
                    "tspFromClient": get_timestamp(),
                }
            )
            headers = {
                "User-Agent": msToken_config["User-Agent"],
                "Content-Type": "application/json",
            }
            url = msToken_config["url"]
            response = requests.post(url, data=payload, headers=headers,proxies=proxies)

            msToken = str(httpx.Cookies(response.cookies).get("msToken"))
            douyin_logger.info("获取msToken......")
            print(response.json())
            print(msToken)
            douyin_logger.info(f"msToken长度:{len(msToken)}")
            if len(msToken) not in [120,128]:
                retries += 1
                print("msToken长度错误")
                continue
            douyin_logger.info("msToken获取成功......")
            return msToken
        except (ProxyError, ConnectionError, Timeout, SSLError) as e:
            douyin_logger.warning(f"Proxy connection failed ({type(e).__name__}): {str(e)}")
            retries += 1
            continue
        except Exception as e:
            douyin_logger.error(f"Unexpected error: {str(e)}", exc_info=True)
            retries  += 1
            continue
    return get_false_msToken()

def get_false_msToken():
    return gen_random_str(126) + "=="
def gen_random_str(randomlength: int) -> str:
    base_str = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-"
    return "".join(random.choice(base_str) for _ in range(randomlength))


def is_more_than_180_days(time_str, time_format="%Y-%m-%d %H:%M:%S"):
    """
    判断给定时间字符串与当前时间的间隔是否大于180天
    """
    try:
        # 将字符串转换为datetime对象
        given_time = datetime.strptime(time_str, time_format)

        # 获取当前时间
        current_time = datetime.now()

        # 计算时间差
        time_difference = current_time - given_time

        # 判断是否大于180天
        return time_difference > timedelta(days=180)
    except ValueError as e:
        print(f"时间字符串格式错误: {e}")
        return False

def is_valid_user(sec_uid):
    post_list = get_first_post_by_sec_uid(sec_uid)
    if post_list == []:
        return False

    for post in post_list:
        create_time = post["create_time"]
        if not is_more_than_180_days(create_time):
            return True

    douyin_logger.info(f"User {sec_uid} is more than 180 days, skipping...")
    return False

def task_douyin(now):
    global STOP_THREAD
    global TTWID
    try:
        TTWID = feishu_api.get_douyin_ttwid_from_feishu(access_token,app_token,sheet_keyword_list_douyin)
        STOP_THREAD = False
        thread_update()
        task_keyword_list_douyin_full_update()
        insert_success_log_by_media("douyin")
        message = f"✅ DouYin脚本执行完成\n开始时间：{THIS_UPDATE_TIME}\n当前时间：{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n本次执行更新了{THIS_UPDATE_KEYWORD_NUM_DY}个关键词:{THIS_UPDATE_KEYWORDS}\n本次执行更新了{THIS_UPDATE_CREATOR_NUM_DY}个作者，{THIS_UPDATE_VIDEO_NUM_DY}个作品\n耗时：{time.perf_counter() - now}秒"
        print(message)
        feishu_api.sendToFeishu(message)
    finally:
        STOP_THREAD = True


def get_video_detail():
    retries = 0
    while retries < 3:
        proxy = get_random_proxy()
        douyin_logger.info(f"Using proxy: {proxy}")
        try:
            proxy_url = f"http://{proxy['ip']}:{proxy['port']}"
            proxies = {'http': proxy_url, 'https': proxy_url}
            api = "/aweme/v1/play/?aweme_id=7505719464839662907& HTTP/2"
            url = "https://api-hl.amemv.com"
            headers = {

                "X-Tt-Dt": "AAATKP4WRZUKJVEZIAVQIIMXMUC63GWMAP5KVOFAK4OY5DFF5HFVXI33GJP27QTTXGYEBMTSREIYUIHFODGU4X7N5T3ITAEEGL6RUG5VNHKQENA5WBFBR3YBZLLVE",
                "X-Common-Params-V2" : "ac=wifi&aid=1128&appTheme=light&app_name=aweme&app_type=normal&cdid=84b4bc17-c80e-409d-9a5b-d8e1548a805b&channel=ali_1128_64&cpu_support64=true&device_brand=samsung&device_id=529294601252441&device_platform=android&device_type=SM-G9910&dpi=480&host_abi=arm64-v8a&iid=2569988447928768&is_android_pad=0&is_guest_mode=0&language=zh&manifest_version_code=270801&minor_status=0&os=android&os_api=32&os_version=12&resolution=1440*2560&ssmix=a&update_version_code=27809900&version_code=270800&version_name=27.8.0",
                "X-Medusa": "zjhBaPpU1AT5ShvzKOxcDQSAOE5jUHZ1H5zSldqsNOURCFX2f93m0sH3A33bJkhq2QLFjGzvhQX/bEwj+WyPjt7RWWm7m8s/dYwbK4yJ+oD6KPaVwK58KRWvRXvDsG9KAa6vnFS28ytCgQUozrEbz54ypvGP4O0TeFO0OQI+Pl96TKaiSe7ix9GLoJseiilG2SWBryAdMYmrV8kGAIOe2Jy/WVvtea/MC8mrsTSrxtq/TvjxmjKDYfdn0iBLt61Cvi7RseUIAKpjW/Hm+nbaAP4EdzXYTtcMkfDWSh+mMLu1pSJhbNhmf1BS4X8PWoXZiqQhtLSHmMN6ETM+E9DbEUkJDfuXwRSDggyqpHs2noE7chYhXNV+X0YQ1GNiPuQgJHnsW/42NBYnDEsbuodgKEmEZavmBzftqUosmCy7txZFSO+z0WD1Jqm9lhlQ1p40rBMv+48wdM5m/BE09wUR97RCwIHeGS4n6PwZpz5Ldzy4FnWwJA24n3EhnibTr6uvZ+xRRXXGpFTkK00r+Qps431U7VdFDI5V4+OWQMZKBSiAJYDClItVPgzRu0gP8r29RBOuuG0XVVAmWL3pQffgY54W2bIhY3f8ny7yuMBcEmKSIgXGp+92aPxRIgp/vzI4+ZBkWYfxAwPKypOaW6shimyMsSpbvG47sHZejMgppKtVmSlMr14cyywiLLvALhBxD22kcpjcvpqppTqB97YOXF7+KTjOwFp0YJNHwDJkp/1dPL8UNOOojZCE2FCnWx2BvwQkR2NfN7ywJnDLjy9pijceZTEqhzyEN7V99ii2JgJiqCZzWE2892wLIixtZZAt+ILM47W4qAt40jfLbAV8b14hdFSdzn+3c3Rrycx7xCskFdhCJCCi7xFSNtLp+EjJfYamyu1hr6nFbxvE7K/gTwyUq9r7V0rP6Sc5+iai5Q9VlDlUczX2LSo92Vzgcf0znCTTk3qkkm/NT55HPYUJy/wU2uQSJVmZ9x+39nRvFXIIRClQ1+w=",
                "User-Agent": "com.ss.android.ugc.aweme/270801 (Linux; U; Android 12; zh_CN; SM-G9910; Build/W528JS; Cronet/TTNetVersion:eb99db8f 2023-11-08 QuicVersion:43f5661a 2023-09-26)",
                "Cookie" : "passport_csrf_token=a2f549ee4f01aa3f955c9f55c3c7204d; passport_csrf_token_default=a2f549ee4f01aa3f955c9f55c3c7204d; store-region=cn-sh; store-region-src=did; install_id=2569988447928768; ttreq=1$55bebb9a0d25b7bac251c6a066c0ef07f103c569; odin_tt=4b4c67834e3ab4f9039ddaaa8cf79aae506bdbbb5ee109b8955514276e8305095e6eec168ee6c32924a98e81089a4ba013d6608bcbba0d9d1a8323acc19763ddd66ae97ea04fe38671e4e6d159759408"
            }
            response = requests.get(url + api, headers=headers, proxies=proxies, timeout=10)
            print(response.text)
            print(response.status_code)
            print(response.json())
            return 
        except Exception as e:
            douyin_logger.error(f"Error occurred: {e}")
            retries += 1
            
if __name__ == "__main__":
    # get_real_msToken()
    # get_video_detail()
    # thread_update()
    # time.sleep(4)
    # get_top100_posts_by_keyword("街舞")
    # gen_ttwid()
    # get_first_post_by_sec_uid("MS4wLjABAAAANXSltcLCzDGmdNFI2Q_QixVTr67NiYzjKOIP5s03CAE")
    # get_douyin_user_info_v2("MS4wLjABAAAAnQa4b4ej8prkgqVP-J_EQQC-T7QyZEsek666KR7abi_hLBRygAl-AD7DnmZNVdGt")

    # get_first_post_by_sec_uid("MS4wLjABAAAAFtX8dU3X3geAXNu-soxYq-2x8SWyfQMjsh7g7QmXBLofW51RBygWTsCcWUJZG8ki")

    # get_register_time_by_sec_uid("MS4wLjABAAAAFtX8dU3X3geAXNu-soxYq-2x8SWyfQMjsh7g7QmXBLofW51RBygWTsCcWUJZG8ki")
    # get_register_time_by_sec_uid("MS4wLjABAAAAFtX8dU3X3geAXNu-soxYq-2x8SWyfQMjsh7g7QmXBLofW51RBygWTsCcWUJZG8ki")
    # time.sleep(4)

    # get_first_post_by_sec_uid("MS4wLjABAAAAnQa4b4ej8prkgqVP-J_EQQC-T7QyZEsek666KR7abi_hLBRygAl-AD7DnmZNVdGt")
    # get_register_time_by_sec_uid("MS4wLjABAAAAnQa4b4ej8prkgqVP-J_EQQC-T7QyZEsek666KR7abi_hLBRygAl-AD7DnmZNVdGt")
    # get_posts_by_sec_uid("MS4wLjABAAAAnQa4b4ej8prkgqVP-J_EQQC-T7QyZEsek666KR7abi_hLBRygAl-AD7DnmZNVdGt")


 



    # time.sleep(5)

    task_douyin(time.perf_counter())



